#!/usr/bin/env python
# encoding: utf-8

import urllib2
import gzip
from StringIO import StringIO
from lxml import html
import json
import csv
from multiprocessing import Pool

def fetch_url(url):
    '''加上gzip请求头接受gzip请求，可加速'''
    # attempt to fetch the URL's contents with gzip compression
    request = urllib2.Request(url)
    request.add_header('Accept-encoding', 'gzip')
    request.add_header('User-Agent', 
        'Mozilla/5.0 (Windows NT 6.1; WOW64) \
        AppleWebKit/537.36 (KHTML, like Gecko) \
        Chrome/36.0.1985.125 Safari/537.36')
    #请求失败时尝试重试一次
    try:
        response = urllib2.urlopen(request, timeout=20)
    except:
        response = urllib2.urlopen(request, timeout=20)
    # get ready for the content
    content = ''
    # if the response is gzip-encoded as expected
    if response.info().get('Content-Encoding') == 'gzip':
      # read the encoded response into a buffer
      buffer = StringIO(response.read())
      # gzip decode the response
      f = gzip.GzipFile(fileobj=buffer)
      # store the result
      content = f.read()
      # close the buffer
      buffer.close()
    # else if the response isn't gzip-encoded
    else:
      # store the result
      content = response.read()
    # return the content
    return content

def get_detail_car_info(url):
    '''获取汽车详情，需要一个参数url，为汽车详情页'''
    car_info = {}
    car_info_page_raw = fetch_url(url).decode('gbk','replace')
    tree = html.fromstring(car_info_page_raw)

    try:
        image = tree.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[1]/div/div[1]/a/img/@src')[0] #预览图片
    except:
        image = tree.xpath('/html/body/div[3]/div[4]/div/div[2]/div[1]/dl[1]/dt/a/img/@src')[0]
    try:
        price = tree.xpath('/html/body/div/div/div/div/div/dl/dt/a[1]/text()')[0] #新车指导价
    except:
        try:
            price = tree.xpath('/html/body/div[3]/div[4]/div/div[1]/div[1]/span[2]/strong/text()')[0]
        except:
            price = tree.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[1]/dl/dt[1]/span/text()')[0]
    engine = tree.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[1]/dl/dd[2]/a/text()') or \
            tree.xpath('/html/body/div[3]/div[4]/div/div[2]/div[1]/dl[2]/dd/span/text()') #发动机
    gearbox = tree.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[1]/dl/dd[3]/span/preceding-sibling::a/text()') or \
            tree.xpath('/html/body/div[3]/div[4]/div/div[2]/div[1]/dl[3]/dd[1]/span/text()') #变速箱
    structure = tree.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[1]/dl/dd[3]/span/following-sibling::a/text()') or \
            tree.xpath('/html/body/div[3]/div[4]/div/div[2]/div[1]/dl[3]/dd[2]/span/text()') #车身架构
    color = tree.xpath('/html/body/div/div/div/div/div/dl/dd/div/a/div/div/text()') or \
            tree.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[1]/dl/dd[1]/span[2]/text()') #车身颜色
    
    car_info['image'] = image
    car_info['price'] = price
    car_info['color'] = color
    car_info['engine'] = engine
    car_info['gearbox'] = gearbox
    car_info['structure'] = structure
    return car_info

def parse_cars_detail_info(cars_detail_info):
    '''解析输出car_detail_info，需要一个列表类型'''
    for car_group in cars_detail_info:
        print '==================================='
        print u'分组名: ' + car_group['Name']
        print '------------------------------'
        for car in car_group['SeriesList']:
            print u'    汽车名: ' + car['Name']
            print u'    详情页: ' + car['url']
            print u'    详细参数: '
            car_info = car['car_info']
            print u'        图片: ' + car_info['image']
            print u'        指导价: ' + car_info['price']
            print u'        车身颜色: ' + ' '.join(car_info['color'])
            print u'        发动机: ' + ' '.join(car_info['engine'])
            print u'        变速箱: ' + ' '.join(car_info['gearbox'])
            print u'        车身结构: ' + ' '.join(car_info['structure'])
            print '------------------------------'

def xml_cars_detail_info(cars_detail_info):
    '''将cars_detsil_info列表转换为xml形式'''
    from lxml import etree
    root_node = etree.Element('root')

    for car_group in cars_detail_info:
        group_node = etree.Element('group')
        group_node_name = etree.Element('Name')
        group_node_name.text = car_group['Name']
        serieslist_node = etree.Element('SeriesList')
        
        for car in car_group['SeriesList']:
            car_node = etree.Element('car')

            name_node = etree.Element('Name')
            name_node.text = car['Name']
            url_node = etree.Element('url')
            url_node.text = car['url']
            
            car_info_node = etree.Element('car_info')
            car_info = car['car_info']
            image_node = etree.Element('image')
            image_node.text = car_info['image']
            price_node = etree.Element('price')
            price_node.text = car_info['price']
            color_node = etree.Element('color')
            color_node.text = ' '.join(car_info['color'])
            engine_node = etree.Element('engine')
            engine_node.text = ' '.join(car_info['engine'])
            gearbox_node = etree.Element('gearbox')
            gearbox_node.text = ' '.join(car_info['gearbox'])
            structure_node = etree.Element('structure')
            structure_node.text = ' '.join(car_info['structure'])
            
            map(car_info_node.append, [image_node,price_node,color_node,engine_node,gearbox_node,structure_node])
            map(car_node.append, [name_node,url_node,car_info_node])
            serieslist_node.append(car_node)
        
        map(group_node.append,[group_node_name,serieslist_node])
        root_node.append(group_node)
        
    print u'写入xml文件...'
    etree.ElementTree(root_node).write('temp.xml',encoding='utf-8',pretty_print=True, xml_declaration=True)
    print '--------------------'

def csv_cars_detail_info(cars_detail_info):
    '''将cars_detsil_info列表转换为csv形式'''
    lines_list = [['汽车名', '指导价', '车身颜色', '图片',\
            '发动机', '变速箱', '车身结构', '详情页']]
    for car_groups in cars_detail_info:
        car_group = car_groups['SeriesList']
        for car in car_group:
            line = []
            car_name = car['Name'].encode('utf8')
            car_info = car['car_info']
            car_price = car_info['price'].encode('utf8') or 'NULL'
            car_color = ' '.join(car_info['color']).encode('utf8') or 'NULL'
            car_image = car_info['image']
            car_engine = ' '.join(car_info['engine']).encode('utf8') or 'NULL'
            car_gearbox = ' '.join(car_info['gearbox']).encode('utf8') or 'NULL'
            car_structure = ' '.join(car_info['structure']).encode('utf8') or 'NULL'
            car_url = car['url']
            map(line.append, \
                [car_name, car_price, car_color, car_image, car_engine,\
                car_gearbox, car_structure, car_url])
            lines_list.append(line)
    
    print u'写入csv文件'
    with open('temp.csv','wb') as csvfile:
        csv.writer(csvfile,quoting=csv.QUOTE_MINIMAL).writerows(lines_list)
    print '------------------------'

def get_homepage_cars_info():
    '''获取首页的汽车信息，返回列表'''
    homepage = fetch_url('http://www.autohome.com.cn/').decode('gbk')
    tree = html.fromstring(homepage)
    cars_url_list = []
    url_count = 0
    cars_detail_url_list = []

    #获取热门车/SUV
    suv_list = tree.xpath('/html/body/div/div/div/div/div/div/dl[1]/dd/ul/li/div/a[not(@class)]')
    #获取热门车/紧凑车型
    compact_car_list = tree.xpath('/html/body/div/div/div/div/div/div/dl[2]/dd/ul/li/div/a[not(@class)]')
    #获取热门车/中型车
    midsize_car_list = tree.xpath('/html/body/div/div/div/div/div/div/dl[3]/dd/ul/li/div/a[not(@class)]')
    servieslist_suv = []
    for suv in suv_list:
        name = suv.text
        url = 'http://www.autohome.com.cn' + suv.attrib['href']
        url = url.split('#')[0]
        cars_detail_url_list.append(url)
        url_count += 1
        servieslist_suv.append({"Name": name, "url": url})

    servieslist_compact_car = []
    for compact_car in compact_car_list:
        name = compact_car.text
        url = 'http://www.autohome.com.cn' + compact_car.attrib['href']
        url = url.split('#')[0]
        cars_detail_url_list.append(url)
        url_count += 1
        servieslist_compact_car.append({"Name": name, "url": url})

    servieslist_midsize_car = []
    for midsize_car in midsize_car_list:
        name = midsize_car.text
        url = 'http://www.autohome.com.cn' + midsize_car.attrib['href']
        url = url.split('#')[0]
        cars_detail_url_list.append(url)
        url_count += 1
        servieslist_midsize_car.append({"Name": name, "url": url})
    
    cars_url_list.append({"Name": u"SUV", "SeriesList": servieslist_suv})
    cars_url_list.append({"Name": u"紧凑车型", "SeriesList": servieslist_compact_car})
    cars_url_list.append({"Name": u"中型车", "SeriesList": servieslist_midsize_car})

    #获取其他分组
    cars_json_raw = fetch_url('http://www.autohome.com.cn/ashx/AjaxIndexHotCar.ashx').decode('gbk')
    cars_dict = json.loads(cars_json_raw)
    #print cars_dict
    for car_group in cars_dict:
        name = car_group['Name']
        serieslist = []
        for i in car_group['SeriesList']:
            i_name = i['Name']
            i_url = 'http://www.autohome.com.cn/' + str(i['Id'])
            url_count += 1
            cars_detail_url_list.append(i_url)
            serieslist.append({"Name": i_name, "url": i_url})
        cars_url_list.append({"Name": name, "SeriesList": serieslist})

    return cars_url_list, url_count, cars_detail_url_list

def main():
    cars_detail_info = []
    cars_info, url_count, cars_detail_url_list = get_homepage_cars_info()
    pool = Pool(processes=10)

    print u'正在获取信息，请稍后...'
    cars_detail_list = pool.map_async(get_detail_car_info, cars_detail_url_list).get(120)
    i = 0
    for car_group in cars_info:
        car_group_name = car_group['Name']
        car_group_serieslist = []
        for car in car_group['SeriesList']:
            car_name = car['Name']
            car_url = car['url']
            car_info = cars_detail_list[i]
            car_group_serieslist.append({'Name': car_name, 'url': car_url, 'car_info': car_info})
            i += 1
        cars_detail_info.append({"Name": car_group_name, "SeriesList": car_group_serieslist})

    parse_cars_detail_info(cars_detail_info)
    xml_cars_detail_info(cars_detail_info)
    csv_cars_detail_info(cars_detail_info)

if __name__ == '__main__':
    main()
